#!/usr/bin/env python3

'''
    do data preprocessing
'''
import sys
sys.path.append(r'../config')
from config import *
import pymongo
from multiprocessing import Pool

#setting mongodb info
client = pymongo.MongoClient(MONGO_URL)
#collection paper_program_paper_info
db_paper_info = client[MONGO_DB]
#collection paper_program_data_info
db_data_info = client[MONGO_DATA_INFO]

'''
    save all researcher name and afflication info
'''
def save_all_name_and_afflication_info(university_name):
    university_data = db_paper_info[university_name].find({}).batch_size(1)
    for data_item in university_data:
        if data_item.get("F"):
            for field_item in data_item.get("F"):
                print(field_item)
                db_data_info['all_field_info'].save(field_item)

        if data_item.get("AA"):
            for researcher_item in data_item["AA"]:
                #set researcher sex info
                researcher_item["Sex"] = 1
                print(researcher_item)
                db_data_info["all_researcher_info"].save(researcher_item)

'''
    save all paper info
'''
def save_all_paper_info(university_name):
    university_data = db_paper_info[university_name].find({}).batch_size(1)
    for data_item in university_data:
        print(data_item)
        db_data_info['all_paper_info'].save(data_item)

'''
    get all paper number
'''
def get_all_paper_number():
    paper_length = 0
    for university_name in UNIVERSITY_LIST1:
        paper_length += db_paper_info[university_name].find({}).count()

    print(paper_length)

'''
    save deletion of duplicated researcher name and afflication info
'''
def save_duplicated_name_and_afflication_info():
    field_name_list = []
    researcher_name_list = []
    field_data_info = db_data_info['all_field_info'].find({}).batch_size(1)
    researcher_data_info = db_data_info["all_researcher_info"].find({}).batch_size(1)

    #list(field_data_info)
    #field_data_info = list(set(list(field_data_info)))
    #researcher_data_info = list(set(list(researcher_data_info)))
    #for field_item in field_data_info:
    #    db_data_info["duplicated_field_info"].save(field_item)

#    for researcher_item in researcher_data_info:
#        db_data_info["duplicated_researcher_info"].save(researcher_item)



    for field_data_item in field_data_info:
        if field_data_item in field_name_list:
            pass
        else:
            field_name_list.append(field_data_item)
            db_data_info["duplicated_field_info"].save(field_data_item)
    for researcher_data_item in researcher_data_info:
        if researcher_data_item in researcher_name_list:
            pass
        else:
            pass



'''
    匹配每篇文章领域
'''
def match_papers_field(paper_item):

    # for paper_item in papers_data_info:
    ori_filed_list = []
    field_list = paper_item.get("F")
    print(field_list)
    for field_item in field_list:
        ori_field = db_data_info['matched_field_name_info'].find({"name":field_item.get("FN")})
        for ori_item in ori_field:
            ori_filed_list.append(ori_item.get("ori_name"))
            print(ori_filed_list)
        print("~~~~~~")
    return list(set(ori_filed_list))

def main():
    #save all researcher name and afflication info
    pool = Pool(processes = 10)
    #pool.map(save_all_name_and_afflication_info,UNIVERSITY_LIST1)
    #get_all_paper_number()
    # pool.map(save_all_paper_info,UNIVERSITY_LIST1)

    # save_duplicated_name_and_afflication_info()

    papers_data_info = db_data_info['all_paper_info'].find({})
    # papers_data_info = list(papers_data_info)
    papers_field_list = pool.map(match_papers_field,papers_data_info)

    for paper_field_list in papers_field_list:
        for field_item in paper_field_list:
            if field_item == "Biology":
                Biology_number += 1
            elif field_item == "Physics":
                Physics_number += 1
            elif field_item == "Materials science":
                Materials_science_number += 0
            elif field_item == "Chemistry":
                Chemistry_number += 1
            elif field_item == "Environmental science":
                Environmental_science_number +=1
            elif field_item == "Mathematics":
                Mathematics_number += 1
            elif field_item == "Computer science":
                Computer_science_number += 1
            elif field_item == "Psychology":
                Psychology_number += 1
            elif field_item == "Sociology":
                Sociology_number += 1
            elif field_item == "Geology":
                Geology_number += 1
            elif field_item == "Political science":
                Political_science_number += 1
            elif field_item == "Geography":
                Geography_number += 1
            elif field_item == "Medicine":
                Medicine_number += 1
            elif field_item == "History":
                History_number += 1
            elif field_item == "Engineering":
                Engineering_number += 1
            elif field_item == "Philosophy":
                Philosophy_number += 1
            elif field_item == "Economics":
                Economics_number += 1
            elif field_item == "Art":
                Art_number += 1
            elif field_item == "Business":
                Business_number += 1
            else:
                print("出错，因为没匹配到！")
                break

    list = []
    list.append({"Biology":Biology_number})
    list.append({"Physics":Physics_number})
    list.append({"Materials science":Materials_science_number})
    list.append({"Chemistry":Chemistry_number})
    list.append({"Environmental science":Environmental_science_number})
    list.append({"Mathematics":Mathematics_number})
    list.append({"Computer science":Computer_science_number})
    list.append({"Psychology":Psychology_number})
    list.append({"Sociology":Sociology_number})
    list.append({"Geology":Geology_number})
    list.append({"Political science":Political_science_number})
    list.append({"Geography":Geography_number})
    list.append({"Medicine":Medicine_number})
    list.append({"History":History_number})
    list.append({"Engineering":Engineering_number})
    list.append({"Philosophy":Philosophy_number})
    list.append({"Economics":Economics_number})
    list.append({"Art":Art_number})
    list.append({"Business":Business_number})

    print(list)


    # match_papers_field()

if __name__ == '__main__':
    main()
